from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
import time
import os
import pandas as pd

# ChromeDriver本地地址
CHROME_DRIVER_PATH = r"C:\Users\86187\AppData\Local\Programs\Python\Python311\chromedriver.exe"
# 数据集存储地址
DATASET_SAVE_PATH = "./data/water_params.xlsx"


def init_page_dataset():
    chrome_option = Options()
    #chrome_option.add_argument('--headless')
    chrome_option.add_experimental_option("prefs", {"profile.managed_default_content_settings.images": 2})
    browser = webdriver.Chrome(options=chrome_option, service=Service(executable_path=CHROME_DRIVER_PATH))
    print("-----------打开网页中--------------")
    browser.get("https://szzdjc.cnemc.cn:8070/GJZ/Business/Publish/RealDatas.html")
    print("-----------网页响应中--------------")
    # 等待界面元素加载加载
    WebDriverWait(browser, 25).until(EC.visibility_of_element_located((By.ID, "ddm_Area")))
    # 模拟点击“区域”
    browser.find_element(By.ID, "ddm_Area").click()
    print('----------获取分类按钮成功-------------')
    # 模拟点击“所有区域”/html/body/div[1]/div[1]/div/ul/li[1]
    browser.find_element(By.XPATH, "/html/body/div[1]/div[1]/div/ul/li[1]").click()
    print('----------切换全国数据成功-------------')
    print("-----------正在解析数据源------------")
    for i in range(30):
        print(f"\r-----------正在解析数据源,当前进度 {round((i / 30 * 100 if i<29 else 100.0), 1)}%------------", end=('' if i<29 else '\n'))
        time.sleep(1)
    soup = BeautifulSoup(browser.page_source, 'html.parser')
    data = soup.find("ul", attrs={'id': 'gridDatas'})
    # 先获取表头数据
    head = soup.find('table', attrs={'id': 'gridHd'}).find_all('td')
    head_params = [item.text for item in head]
    # 再获取当前数据
    content = data.find_all('li', attrs={'class': 'news-item'})
    content_params = []
    for row in content:
        item = row.find('tr').find_all('td')
        content_params.append([column.text for column in item])
    df = pd.DataFrame(data=content_params, columns=head_params)
    target = None
    if os.path.exists(DATASET_SAVE_PATH):
        target = pd.read_excel(DATASET_SAVE_PATH)
    df = pd.concat([target, df])
    df.to_excel(DATASET_SAVE_PATH, index=False, engine='openpyxl')


if __name__ == "__main__":
    init_page_dataset()